SonarQube Issues

Looking at the data

it looks like there might be a difference between the high and low debt groups, the variance is decidedly higher in the high debt group.

d.both_completed %>%
  ggplot(aes(x=sonarqube_issues, fill=high_debt_version)) + 
  geom_boxplot()

mean(d.both_completed$sonarqube_issues)

## [1] 1.659091

var(d.both_completed$sonarqube_issues)

## [1] 5.997357

Descriptive Statistics:

d.both_completed %>%
  pull(sonarqube_issues) %>% 
  summary()

##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   0.000   0.000   1.000   1.659   3.000  12.000

sprintf("Variance: %.2f", var(pull(d.both_completed, sonarqube_issues)))

## [1] "Variance: 6.00"

Initial model

Variable names are modeled using the negative binomial family rather than poisson since the variance is much greater than the mean.

We include high_debt_verison as a predictor in our model as this variable represent the very effect we want to measure. We also include a varying intercept for each individual to prevent the model from learning too much from single participants with extreme measurements.

Selecting Priors

We iterate over the model until we have sane priors, in this case a prior giving a 50/50 chance was chosen in both cases.

sonarqube_issues.with <- extendable_model(
  base_name = "sonarqube_issues",
  base_formula = "sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
  base_priors = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape")
  ),
  family = negbinomial(),
  data = d.both_completed,
  base_control = list(adapt_delta = 0.98)
)

Default priors

prior_summary(sonarqube_issues.with(only_priors= TRUE))

Selected priors

prior_summary(sonarqube_issues.with(sample_prior = "only"))

Prior Predictive Check

pp_check(sonarqube_issues.with(sample_prior = "only"), nsamples = 200, type = "bars")  + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

Beta Parameter Influence

sim.size <- 1000
sim.intercept <- rnorm(sim.size, 1.5, 1)
sim.beta <- rnorm(sim.size, 0, 1)
sim.beta.diff <- exp(sim.intercept + sim.beta) - exp(sim.intercept)
sim.beta.diff.min <- sim.beta.diff

data.frame(x = sim.beta.diff.min) %>%
  ggplot(aes(x)) +
  geom_density() +
  xlim(-10, 10) +
  labs(
    title = "Beta parameter prior influence",
    x = "Issues difference difference (min)",
    y = "Density"
  )

Model fit

We check the posterior distribution and can see that the model seems to have been able to fit the data well Sampling seems to also have worked well as Rhat values are close to 1 and the sampling plots look nice. #### Posterior Predictive check

pp_check(sonarqube_issues.with(), nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 29 rows containing missing values (geom_pointrange).

#### Summary

summary(sonarqube_issues.with())

## Warning: There were 1 divergent transitions after warmup. Increasing adapt_delta
## above may help. See http://mc-stan.org/misc/warnings.html#divergent-transitions-
## after-warmup

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) 
##    Data: as.data.frame(data) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.49      0.35     0.02     1.28 1.00      737     1649
## 
## Population-Level Effects: 
##                        Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                  0.75      0.33     0.09     1.39 1.00     2283
## high_debt_versionfalse    -0.67      0.43    -1.48     0.20 1.00     4800
##                        Tail_ESS
## Intercept                  2787
## high_debt_versionfalse     3450
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.88     14.63     0.33     4.67 1.01      785      532
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Sampling plots

plot(sonarqube_issues.with(), ask = FALSE)

Model predictor extenstions

# default prior for monotonic predictor
edlvl_prior <- prior(dirichlet(2), class = "simo", coef = "moeducation_level1")

One variable

loo_result <- loo(
  # Benchmark model(s)
  sonarqube_issues.with(),
  # New model(s)
  sonarqube_issues.with("modified_lines"),
  sonarqube_issues.with("work_domain"),
  sonarqube_issues.with("work_experience_programming.s"),
  sonarqube_issues.with("work_experience_java.s"),
  sonarqube_issues.with("education_field"),
  sonarqube_issues.with("mo(education_level)", edlvl_prior),
  sonarqube_issues.with("workplace_peer_review"),
  sonarqube_issues.with("workplace_td_tracking"),
  sonarqube_issues.with("workplace_pair_programming"),
  sonarqube_issues.with("workplace_coding_standards"),
  sonarqube_issues.with("scenario"),
  sonarqube_issues.with("group")
)

Comparison

loo_result[2]

## $diffs
##                                                           elpd_diff se_diff
## sonarqube_issues.with("group")                             0.0       0.0   
## sonarqube_issues.with("work_domain")                      -0.6       2.1   
## sonarqube_issues.with()                                   -1.0       1.5   
## sonarqube_issues.with("workplace_coding_standards")       -1.2       1.4   
## sonarqube_issues.with("mo(education_level)", edlvl_prior) -1.4       1.4   
## sonarqube_issues.with("scenario")                         -1.4       1.8   
## sonarqube_issues.with("workplace_peer_review")            -1.6       1.3   
## sonarqube_issues.with("workplace_pair_programming")       -1.6       1.4   
## sonarqube_issues.with("workplace_td_tracking")            -1.7       1.5   
## sonarqube_issues.with("modified_lines")                   -1.9       1.5   
## sonarqube_issues.with("work_experience_java.s")           -2.3       1.4   
## sonarqube_issues.with("work_experience_programming.s")    -2.4       1.6   
## sonarqube_issues.with("education_field")                  -2.5       1.7

Diagnostics

loo_result[1]

## $loos
## $loos$`sonarqube_issues.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.8  7.8
## p_loo         6.7  1.3
## looic       155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   560       
##  (0.5, 0.7]   (ok)       13    29.5%   112       
##    (0.7, 1]   (bad)       3     6.8%   37        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("modified_lines")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.7  7.8
## p_loo         6.8  1.2
## looic       157.4 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     33    75.0%   727       
##  (0.5, 0.7]   (ok)        7    15.9%   184       
##    (0.7, 1]   (bad)       4     9.1%   105       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("work_domain")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.4  7.6
## p_loo         8.7  1.1
## looic       154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   1137      
##  (0.5, 0.7]   (ok)       14    31.8%   483       
##    (0.7, 1]   (bad)       2     4.5%   1173      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("work_experience_programming.s")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -79.3  8.4
## p_loo         7.9  2.0
## looic       158.5 16.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     30    68.2%   403       
##  (0.5, 0.7]   (ok)       12    27.3%   123       
##    (0.7, 1]   (bad)       2     4.5%   20        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("work_experience_java.s")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -79.1  8.2
## p_loo         7.7  1.8
## looic       158.2 16.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     33    75.0%   981       
##  (0.5, 0.7]   (ok)        7    15.9%   185       
##    (0.7, 1]   (bad)       4     9.1%   52        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("education_field")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -79.3  8.1
## p_loo         8.9  1.7
## looic       158.6 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     30    68.2%   38        
##  (0.5, 0.7]   (ok)        7    15.9%   138       
##    (0.7, 1]   (bad)       7    15.9%   5         
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.2  7.7
## p_loo         6.9  1.1
## looic       156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     29    65.9%   533       
##  (0.5, 0.7]   (ok)       11    25.0%   178       
##    (0.7, 1]   (bad)       4     9.1%   141       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_peer_review")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.4  8.0
## p_loo         7.3  1.4
## looic       156.8 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     24    54.5%   617       
##  (0.5, 0.7]   (ok)       18    40.9%   232       
##    (0.7, 1]   (bad)       2     4.5%   78        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_td_tracking")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.5  7.9
## p_loo         7.4  1.3
## looic       157.0 15.8
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     29    65.9%   605       
##  (0.5, 0.7]   (ok)       11    25.0%   122       
##    (0.7, 1]   (bad)       4     9.1%   162       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_pair_programming")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.4  7.9
## p_loo         7.2  1.3
## looic       156.9 15.8
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     29    65.9%   459       
##  (0.5, 0.7]   (ok)       10    22.7%   91        
##    (0.7, 1]   (bad)       5    11.4%   146       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.0  8.1
## p_loo         8.1  1.7
## looic       156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     27    61.4%   363       
##  (0.5, 0.7]   (ok)       12    27.3%   272       
##    (0.7, 1]   (bad)       5    11.4%   52        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("scenario")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.2  8.1
## p_loo         8.2  1.6
## looic       156.5 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   467       
##  (0.5, 0.7]   (ok)       12    27.3%   113       
##    (0.7, 1]   (bad)       4     9.1%   287       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("group")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -76.8  7.7
## p_loo         7.5  1.2
## looic       153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     32    72.7%   503       
##  (0.5, 0.7]   (ok)       10    22.7%   233       
##    (0.7, 1]   (bad)       2     4.5%   106       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Two variables

loo_result <- loo(
  # Benchmark model(s)
  sonarqube_issues.with(),
  sonarqube_issues.with("group"),
  sonarqube_issues.with("work_domain"),
  sonarqube_issues.with("workplace_coding_standards"),
  sonarqube_issues.with("mo(education_level)", edlvl_prior),
  #New model(s)
  sonarqube_issues.with(c("work_domain","workplace_coding_standards")),
  sonarqube_issues.with(c("mo(education_level)","work_domain"), edlvl_prior),
  sonarqube_issues.with(c("workplace_coding_standards","mo(education_level)"), edlvl_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                                                elpd_diff
## sonarqube_issues.with("group")                                                                  0.0     
## sonarqube_issues.with("work_domain")                                                           -0.6     
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards"))                          -0.8     
## sonarqube_issues.with()                                                                        -1.0     
## sonarqube_issues.with("workplace_coding_standards")                                            -1.2     
## sonarqube_issues.with("mo(education_level)", edlvl_prior)                                      -1.4     
## sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"),     edlvl_prior) -1.4     
## sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)                -1.5     
##                                                                                                se_diff
## sonarqube_issues.with("group")                                                                  0.0   
## sonarqube_issues.with("work_domain")                                                            2.1   
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards"))                           2.0   
## sonarqube_issues.with()                                                                         1.5   
## sonarqube_issues.with("workplace_coding_standards")                                             1.4   
## sonarqube_issues.with("mo(education_level)", edlvl_prior)                                       1.4   
## sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"),     edlvl_prior)  1.4   
## sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)                 2.0

Diagnostics

loo_result[1]

## $loos
## $loos$`sonarqube_issues.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.8  7.8
## p_loo         6.7  1.3
## looic       155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   560       
##  (0.5, 0.7]   (ok)       13    29.5%   112       
##    (0.7, 1]   (bad)       3     6.8%   37        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("group")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -76.8  7.7
## p_loo         7.5  1.2
## looic       153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     32    72.7%   503       
##  (0.5, 0.7]   (ok)       10    22.7%   233       
##    (0.7, 1]   (bad)       2     4.5%   106       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("work_domain")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.4  7.6
## p_loo         8.7  1.1
## looic       154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   1137      
##  (0.5, 0.7]   (ok)       14    31.8%   483       
##    (0.7, 1]   (bad)       2     4.5%   1173      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.0  8.1
## p_loo         8.1  1.7
## looic       156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     27    61.4%   363       
##  (0.5, 0.7]   (ok)       12    27.3%   272       
##    (0.7, 1]   (bad)       5    11.4%   52        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.2  7.7
## p_loo         6.9  1.1
## looic       156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     29    65.9%   533       
##  (0.5, 0.7]   (ok)       11    25.0%   178       
##    (0.7, 1]   (bad)       4     9.1%   141       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with(c("work_domain", "workplace_coding_standards"))`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.6  8.0
## p_loo         9.6  1.7
## looic       155.2 16.0
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   494       
##  (0.5, 0.7]   (ok)        9    20.5%   456       
##    (0.7, 1]   (bad)       1     2.3%   131       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.3  7.7
## p_loo         9.2  1.1
## looic       156.7 15.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     30    68.2%   1281      
##  (0.5, 0.7]   (ok)       11    25.0%   713       
##    (0.7, 1]   (bad)       3     6.8%   203       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with(c("workplace_coding_standards", "mo(education_level)"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.2  8.0
## p_loo         8.3  1.6
## looic       156.5 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   642       
##  (0.5, 0.7]   (ok)       13    29.5%   77        
##    (0.7, 1]   (bad)       3     6.8%   289       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Three variables

loo_result <- loo(
  # Benchmark model(s)
  sonarqube_issues.with(),
  sonarqube_issues.with("group"),
  sonarqube_issues.with("work_domain"),
  sonarqube_issues.with("workplace_coding_standards"),
  sonarqube_issues.with("mo(education_level)", edlvl_prior),
  sonarqube_issues.with(c("mo(education_level)","work_domain"), edlvl_prior),
  #New model(s)
  sonarqube_issues.with(c("work_domain","workplace_coding_standards","mo(education_level)"), edlvl_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                                                               elpd_diff
## sonarqube_issues.with("group")                                                                                 0.0     
## sonarqube_issues.with("work_domain")                                                                          -0.6     
## sonarqube_issues.with()                                                                                       -1.0     
## sonarqube_issues.with("workplace_coding_standards")                                                           -1.2     
## sonarqube_issues.with("mo(education_level)", edlvl_prior)                                                     -1.4     
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards",     "mo(education_level)"), edlvl_prior) -1.5     
## sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)                               -1.5     
##                                                                                                               se_diff
## sonarqube_issues.with("group")                                                                                 0.0   
## sonarqube_issues.with("work_domain")                                                                           2.1   
## sonarqube_issues.with()                                                                                        1.5   
## sonarqube_issues.with("workplace_coding_standards")                                                            1.4   
## sonarqube_issues.with("mo(education_level)", edlvl_prior)                                                      1.4   
## sonarqube_issues.with(c("work_domain", "workplace_coding_standards",     "mo(education_level)"), edlvl_prior)  1.9   
## sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)                                2.0

Diagnostics

loo_result[1]

## $loos
## $loos$`sonarqube_issues.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.8  7.8
## p_loo         6.7  1.3
## looic       155.7 15.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   560       
##  (0.5, 0.7]   (ok)       13    29.5%   112       
##    (0.7, 1]   (bad)       3     6.8%   37        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("group")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -76.8  7.7
## p_loo         7.5  1.2
## looic       153.6 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     32    72.7%   503       
##  (0.5, 0.7]   (ok)       10    22.7%   233       
##    (0.7, 1]   (bad)       2     4.5%   106       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("work_domain")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -77.4  7.6
## p_loo         8.7  1.1
## looic       154.9 15.3
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     28    63.6%   1137      
##  (0.5, 0.7]   (ok)       14    31.8%   483       
##    (0.7, 1]   (bad)       2     4.5%   1173      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("workplace_coding_standards")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.0  8.1
## p_loo         8.1  1.7
## looic       156.1 16.2
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     27    61.4%   363       
##  (0.5, 0.7]   (ok)       12    27.3%   272       
##    (0.7, 1]   (bad)       5    11.4%   52        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.2  7.7
## p_loo         6.9  1.1
## looic       156.4 15.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     29    65.9%   533       
##  (0.5, 0.7]   (ok)       11    25.0%   178       
##    (0.7, 1]   (bad)       4     9.1%   141       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with(c("mo(education_level)", "work_domain"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.3  7.7
## p_loo         9.2  1.1
## looic       156.7 15.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     30    68.2%   1281      
##  (0.5, 0.7]   (ok)       11    25.0%   713       
##    (0.7, 1]   (bad)       3     6.8%   203       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`sonarqube_issues.with(c("work_domain", "workplace_coding_standards",     "mo(education_level)"), edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -78.3  8.0
## p_loo        10.7  1.6
## looic       156.6 15.9
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     32    72.7%   798       
##  (0.5, 0.7]   (ok)       10    22.7%   258       
##    (0.7, 1]   (bad)       2     4.5%   59        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Candidate models

We pick some of our top performing models as candidates and inspect them closer.

The candidate models are named and listed in order of complexity.

Time0

We select the simplest model as a baseline.

Candidate models

We inspect some of our top performing models.

All models seems to have sampled nicely (rhat = 1 and fluffy plots) they also have about the same fit to the data and similar estimates for the high_debt_version beta parameter.

sonarqube_issues0

We select the simplest model as a baseline.

sonarqube_issues0 <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape")
  ),
  family = negbinomial(),
  data = d.both_completed,
  control = list(adapt_delta = 0.97),
  file = "fits/sonarqube_issues0",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues0)

## Warning: There were 1 divergent transitions after warmup. Increasing adapt_delta
## above may help. See http://mc-stan.org/misc/warnings.html#divergent-transitions-
## after-warmup

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) 
##    Data: d.both_completed (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.50      0.35     0.02     1.27 1.00      806     1629
## 
## Population-Level Effects: 
##                        Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                  0.74      0.34     0.07     1.41 1.00     2488
## high_debt_versionfalse    -0.65      0.44    -1.50     0.23 1.00     4294
##                        Tail_ESS
## Intercept                  2271
## high_debt_versionfalse     2762
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.05      1.25     0.31     3.94 1.00      918      807
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues0)

## $session
## , , Intercept
## 
##                             Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95 -0.01245467 0.4727195 -1.0415672 1.0408743
## 6033d90a5af2c702367b3a96 -0.19293194 0.5278476 -1.5095095 0.7067151
## 6034fc165af2c702367b3a98  0.43213782 0.5571518 -0.3137879 1.7727020
## 603500725af2c702367b3a99 -0.14213277 0.5165855 -1.3652020 0.8344129
## 603f97625af2c702367b3a9d  0.44416480 0.5755908 -0.3398858 1.8370410
## 603fd5d95af2c702367b3a9e -0.03703627 0.4691895 -1.0553820 0.9406905
## 60409b7b5af2c702367b3a9f  0.23018380 0.4937926 -0.5627549 1.4366155
## 604b82b5a7718fbed181b336  0.37879216 0.5375917 -0.3401873 1.6490963
## 6050c1bf856f36729d2e5218 -0.15506157 0.5253208 -1.4025005 0.8335826
## 6050e1e7856f36729d2e5219  0.07096857 0.4616443 -0.8227767 1.1993138
## 6055fdc6856f36729d2e521b  0.12884237 0.4656367 -0.7191138 1.2365668
## 60589862856f36729d2e521f -0.31039486 0.6262516 -1.9367860 0.6334938
## 605afa3a856f36729d2e5222 -0.29451717 0.6351819 -1.9435010 0.5985559
## 605c8bc6856f36729d2e5223 -0.01378988 0.4576662 -0.9777489 1.0173585
## 605f3f2d856f36729d2e5224  0.11782844 0.4621454 -0.7215196 1.2256008
## 605f46c3856f36729d2e5225 -0.29320410 0.6277727 -1.9920288 0.6522350
## 60605337856f36729d2e5226 -0.29222811 0.6239834 -1.9377522 0.6389132
## 60609ae6856f36729d2e5228 -0.09936213 0.4662010 -1.1522400 0.8260732
## 6061ce91856f36729d2e522e -0.15275983 0.5156197 -1.4054788 0.7859701
## 6061f106856f36729d2e5231 -0.32119587 0.6658447 -2.0755255 0.6549789
## 6068ea9f856f36729d2e523e -0.01789478 0.4740959 -1.0164057 1.0468773
## 6075ab05856f36729d2e5247  0.06223130 0.4639792 -0.8977303 1.1145690

Sampling plots

plot(sonarqube_issues0, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues0, nsamples = 200, type = "bars")  + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 23 rows containing missing values (geom_pointrange).

sonarqube_issues1

We select the best performing model with one variable.

sonarqube_issues1 <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape")
  ),
  family = negbinomial(),
  data = d.both_completed,
  control = list(adapt_delta = 0.97),
  file = "fits/sonarqube_issues1",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues1)

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain 
##    Data: d.both_completed (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.39      0.31     0.01     1.16 1.00     1368     1824
## 
## Population-Level Effects: 
##                        Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                  0.62      0.44    -0.22     1.52 1.00     3714
## high_debt_versionfalse    -0.72      0.43    -1.55     0.13 1.00     5709
## work_domainAutomotive     -0.42      0.59    -1.58     0.73 1.00     3865
## work_domainDevops         -0.28      0.87    -1.93     1.46 1.00     6487
## work_domainEmbedded       -0.68      0.89    -2.43     1.04 1.00     5973
## work_domainFinance        -0.38      0.68    -1.74     0.96 1.00     4855
## work_domainMixed           0.23      0.74    -1.26     1.73 1.00     5029
## work_domainMusic           0.05      0.79    -1.49     1.58 1.00     4746
## work_domainNone           -0.11      0.58    -1.24     1.03 1.00     4323
## work_domainRetail          0.03      0.75    -1.39     1.50 1.00     5659
## work_domainTelecom         0.34      0.74    -1.10     1.82 1.00     5079
## work_domainWeb             0.98      0.55    -0.16     2.03 1.00     3794
##                        Tail_ESS
## Intercept                  3235
## high_debt_versionfalse     2721
## work_domainAutomotive      2939
## work_domainDevops          3049
## work_domainEmbedded        2966
## work_domainFinance         3035
## work_domainMixed           2747
## work_domainMusic           2921
## work_domainNone            3033
## work_domainRetail          2904
## work_domainTelecom         2782
## work_domainWeb             2848
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.43      4.29     0.35     4.78 1.00     2115     1508
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues1)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.062302805 0.4284255 -0.8038623 1.0963900
## 6033d90a5af2c702367b3a96 -0.086466842 0.4422835 -1.1860380 0.7263208
## 6034fc165af2c702367b3a98  0.139028095 0.4237590 -0.5588491 1.2445150
## 603500725af2c702367b3a99 -0.091043952 0.4396722 -1.1411955 0.7514077
## 603f97625af2c702367b3a9d  0.159626256 0.4263584 -0.5079739 1.2478253
## 603fd5d95af2c702367b3a9e  0.059692768 0.4220291 -0.7530709 1.0847568
## 60409b7b5af2c702367b3a9f  0.236372866 0.4619131 -0.4308832 1.4450993
## 604b82b5a7718fbed181b336  0.109544205 0.4064712 -0.6149302 1.1147643
## 6050c1bf856f36729d2e5218 -0.072452698 0.4679143 -1.2197940 0.8103681
## 6050e1e7856f36729d2e5219  0.034237939 0.4344018 -0.8959063 1.0283230
## 6055fdc6856f36729d2e521b  0.058905157 0.4251294 -0.7987389 1.0847843
## 60589862856f36729d2e521f -0.163521923 0.5111288 -1.5046138 0.6367056
## 605afa3a856f36729d2e5222 -0.190421112 0.5200015 -1.5426623 0.5908613
## 605c8bc6856f36729d2e5223  0.003436533 0.4470740 -0.9731989 1.0178715
## 605f3f2d856f36729d2e5224  0.192214596 0.4540024 -0.5302722 1.3626390
## 605f46c3856f36729d2e5225 -0.187699075 0.4895044 -1.4610958 0.5624858
## 60605337856f36729d2e5226 -0.188267003 0.5109586 -1.5560162 0.5776325
## 60609ae6856f36729d2e5228 -0.028377819 0.4060344 -0.9559951 0.8269868
## 6061ce91856f36729d2e522e -0.214062434 0.4856155 -1.4906555 0.5356961
## 6061f106856f36729d2e5231 -0.209020007 0.5015912 -1.5172275 0.5340190
## 6068ea9f856f36729d2e523e  0.002388252 0.4362535 -0.9695532 0.9812145
## 6075ab05856f36729d2e5247  0.053488114 0.3956305 -0.7228165 1.0120105

Sampling plots

plot(sonarqube_issues1, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues1, nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 43 rows containing missing values (geom_pointrange).

sonarqube_issues2

We select the best performing model with two variables.

sonarqube_issues2 <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level)",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape"),
    edlvl_prior
  ),
  family = negbinomial(),
  data = d.both_completed,
  control = list(adapt_delta = 0.97),
  file = "fits/sonarqube_issues2",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues2)

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level) 
##    Data: d.both_completed (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.41      0.34     0.01     1.25 1.00     1329     2009
## 
## Population-Level Effects: 
##                        Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                  0.68      0.74    -0.73     2.17 1.00     2687
## high_debt_versionfalse    -0.73      0.43    -1.56     0.14 1.00     7346
## work_domainAutomotive     -0.45      0.59    -1.63     0.72 1.00     5165
## work_domainDevops         -0.27      0.84    -1.90     1.41 1.00     7366
## work_domainEmbedded       -0.66      0.87    -2.35     1.07 1.00     7893
## work_domainFinance        -0.39      0.72    -1.78     1.04 1.00     7231
## work_domainMixed           0.25      0.77    -1.28     1.78 1.00     6312
## work_domainMusic           0.03      0.82    -1.58     1.60 1.00     7085
## work_domainNone           -0.12      0.61    -1.33     1.04 1.00     4755
## work_domainRetail          0.03      0.83    -1.60     1.63 1.00     6376
## work_domainTelecom         0.34      0.75    -1.13     1.84 1.00     6205
## work_domainWeb             0.97      0.55    -0.17     2.02 1.00     4680
## moeducation_level         -0.02      0.23    -0.49     0.40 1.00     2526
##                        Tail_ESS
## Intercept                  2498
## high_debt_versionfalse     2908
## work_domainAutomotive      2580
## work_domainDevops          2932
## work_domainEmbedded        3214
## work_domainFinance         3210
## work_domainMixed           2615
## work_domainMusic           3176
## work_domainNone            3237
## work_domainRetail          3243
## work_domainTelecom         3343
## work_domainWeb             3271
## moeducation_level          2633
## 
## Simplex Parameters: 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1]     0.26      0.15     0.03     0.59 1.00     6137
## moeducation_level1[2]     0.24      0.14     0.03     0.56 1.00     7529
## moeducation_level1[3]     0.24      0.14     0.04     0.58 1.00     6943
## moeducation_level1[4]     0.26      0.15     0.04     0.59 1.00     6267
##                       Tail_ESS
## moeducation_level1[1]     2381
## moeducation_level1[2]     2759
## moeducation_level1[3]     3159
## moeducation_level1[4]     2795
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.33      2.81     0.33     4.06 1.00     2238     2251
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues2)

## $session
## , , Intercept
## 
##                             Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.06752982 0.4503658 -0.7710735 1.1634840
## 6033d90a5af2c702367b3a96 -0.08384330 0.4782614 -1.2480452 0.8284552
## 6034fc165af2c702367b3a98  0.14531607 0.4181818 -0.5299412 1.2310248
## 603500725af2c702367b3a99 -0.11004764 0.4803116 -1.2916570 0.8007176
## 603f97625af2c702367b3a9d  0.16619652 0.4532797 -0.5980245 1.3254275
## 603fd5d95af2c702367b3a9e  0.04337720 0.4539804 -0.9270605 1.0913103
## 60409b7b5af2c702367b3a9f  0.24127043 0.5084660 -0.4920085 1.5991810
## 604b82b5a7718fbed181b336  0.12347316 0.4394427 -0.6620433 1.2423290
## 6050c1bf856f36729d2e5218 -0.08069138 0.4792503 -1.2673197 0.8247837
## 6050e1e7856f36729d2e5219  0.02721607 0.4521033 -0.8826569 1.0873413
## 6055fdc6856f36729d2e521b  0.07445620 0.4739312 -0.7951090 1.2236712
## 60589862856f36729d2e521f -0.17862792 0.5366800 -1.5459415 0.7054114
## 605afa3a856f36729d2e5222 -0.20932032 0.5578669 -1.7152450 0.6161624
## 605c8bc6856f36729d2e5223  0.00331001 0.4781140 -1.0297192 1.0413638
## 605f3f2d856f36729d2e5224  0.20850641 0.4860773 -0.5275226 1.5032648
## 605f46c3856f36729d2e5225 -0.17303213 0.5255905 -1.5325325 0.6957678
## 60605337856f36729d2e5226 -0.19324510 0.5434027 -1.5788845 0.6535786
## 60609ae6856f36729d2e5228 -0.04077799 0.4381264 -1.0519090 0.8524392
## 6061ce91856f36729d2e522e -0.24174761 0.5327489 -1.6683308 0.5799865
## 6061f106856f36729d2e5231 -0.22521261 0.5681815 -1.6686407 0.6281218
## 6068ea9f856f36729d2e523e -0.01955714 0.4710601 -1.0435433 0.9995508
## 6075ab05856f36729d2e5247  0.07638554 0.4301520 -0.7741636 1.1343728

Sampling plots

plot(sonarqube_issues2, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues2, nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 47 rows containing missing values (geom_pointrange).

sonarqube_issues3

We select the best performing model with three variables.

sonarqube_issues3 <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level) + workplace_coding_standards",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape"),
    edlvl_prior
  ),
  family = negbinomial(),
  data = d.both_completed,
  control = list(adapt_delta = 0.97),
  file = "fits/sonarqube_issues3",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues3)

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_domain + mo(education_level) + workplace_coding_standards 
##    Data: d.both_completed (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.42      0.33     0.02     1.22 1.00     1232     2067
## 
## Population-Level Effects: 
##                                 Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept                           0.25      0.82    -1.26     1.93 1.00
## high_debt_versionfalse             -0.68      0.42    -1.51     0.19 1.00
## work_domainAutomotive              -0.28      0.59    -1.47     0.86 1.00
## work_domainDevops                  -0.20      0.83    -1.85     1.44 1.00
## work_domainEmbedded                -0.63      0.90    -2.40     1.15 1.00
## work_domainFinance                 -0.44      0.71    -1.84     0.98 1.00
## work_domainMixed                    0.14      0.77    -1.35     1.67 1.00
## work_domainMusic                    0.18      0.77    -1.31     1.65 1.00
## work_domainNone                    -0.28      0.61    -1.48     0.92 1.00
## work_domainRetail                  -0.05      0.82    -1.62     1.58 1.00
## work_domainTelecom                  0.49      0.76    -1.03     1.96 1.00
## work_domainWeb                      0.99      0.56    -0.10     2.05 1.00
## workplace_coding_standardsfalse     0.58      0.50    -0.41     1.55 1.00
## moeducation_level                   0.00      0.22    -0.46     0.41 1.00
##                                 Bulk_ESS Tail_ESS
## Intercept                           2411     2619
## high_debt_versionfalse              5614     2897
## work_domainAutomotive               3854     3209
## work_domainDevops                   4543     3308
## work_domainEmbedded                 6110     2942
## work_domainFinance                  5320     3088
## work_domainMixed                    5350     3412
## work_domainMusic                    5233     3122
## work_domainNone                     3925     3247
## work_domainRetail                   3938     2902
## work_domainTelecom                  4614     2602
## work_domainWeb                      3587     2741
## workplace_coding_standardsfalse     4376     3030
## moeducation_level                   2256     2613
## 
## Simplex Parameters: 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1]     0.26      0.15     0.04     0.59 1.00     5207
## moeducation_level1[2]     0.25      0.14     0.03     0.58 1.00     5297
## moeducation_level1[3]     0.24      0.14     0.03     0.57 1.00     5179
## moeducation_level1[4]     0.25      0.14     0.04     0.56 1.00     5266
##                       Tail_ESS
## moeducation_level1[1]     2991
## moeducation_level1[2]     2669
## moeducation_level1[3]     3213
## moeducation_level1[4]     2904
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.70      4.12     0.37     6.42 1.00     1946     1939
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues3)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.028635020 0.4650921 -0.9529764 1.1047525
## 6033d90a5af2c702367b3a96 -0.056907468 0.4591343 -1.0875695 0.8908031
## 6034fc165af2c702367b3a98  0.075621564 0.4042183 -0.7049300 1.0477538
## 603500725af2c702367b3a99 -0.108523983 0.4787815 -1.3381695 0.8097602
## 603f97625af2c702367b3a9d  0.254238622 0.4909211 -0.4748888 1.4724628
## 603fd5d95af2c702367b3a9e -0.002165308 0.4542532 -1.0026400 1.0046373
## 60409b7b5af2c702367b3a9f  0.235094547 0.5000150 -0.5245863 1.5017948
## 604b82b5a7718fbed181b336  0.048235995 0.4324996 -0.7786700 1.0791168
## 6050c1bf856f36729d2e5218 -0.058117226 0.4816999 -1.1846235 0.8978553
## 6050e1e7856f36729d2e5219  0.018111423 0.4588099 -0.9306302 1.0382193
## 6055fdc6856f36729d2e521b  0.110955132 0.4670653 -0.7539327 1.2592078
## 60589862856f36729d2e521f -0.177640134 0.5437113 -1.6028797 0.7110827
## 605afa3a856f36729d2e5222 -0.185615996 0.5252093 -1.5529320 0.6811358
## 605c8bc6856f36729d2e5223  0.036596625 0.4561369 -0.9061319 1.0759475
## 605f3f2d856f36729d2e5224  0.259323011 0.5212896 -0.4982756 1.6276045
## 605f46c3856f36729d2e5225 -0.188008394 0.5261848 -1.5444333 0.6327229
## 60605337856f36729d2e5226 -0.189419569 0.5585809 -1.5710955 0.7092739
## 60609ae6856f36729d2e5228 -0.048630413 0.4509998 -1.0716695 0.8465676
## 6061ce91856f36729d2e522e -0.202652594 0.5185739 -1.5296473 0.6462789
## 6061f106856f36729d2e5231 -0.232672622 0.5650321 -1.7252327 0.6015934
## 6068ea9f856f36729d2e523e -0.040479170 0.4642363 -1.0705635 0.9746673
## 6075ab05856f36729d2e5247  0.019725881 0.4346092 -0.8879677 1.0071218

Sampling plots

plot(sonarqube_issues3, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues2, nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 42 rows containing missing values (geom_pointrange).

Final Model

All candidate models look nice, none is significantly better than the others, we will proceed the simplest model: time0

Variations

We will try a few different variations of the selected candidate model.

All data points

Some participants did only complete one scenario. Those has been excluded from the initial dataset to improve sampling of the models. We do however want to use all data we can and will therefore try to fit the model with the complete dataset.

sonarqube_issues0.all <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session)",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape")
  ),
  family = negbinomial(),
  data = as.data.frame(d.completed),
  control = list(adapt_delta = 0.97),
  file = "fits/sonarqube_issues0.all",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues0.all)

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) 
##    Data: as.data.frame(d.completed) (Number of observations: 51) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 29) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.47      0.31     0.02     1.15 1.00      988     1925
## 
## Population-Level Effects: 
##                        Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                  0.72      0.30     0.15     1.30 1.00     2383
## high_debt_versionfalse    -0.74      0.38    -1.50     0.03 1.00     5268
##                        Tail_ESS
## Intercept                  2292
## high_debt_versionfalse     2479
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.64      4.02     0.40     6.04 1.00      966      741
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues0.all)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033c6fc5af2c702367b3a93  0.173636078 0.4841949 -0.6685669 1.3959565
## 6033d69a5af2c702367b3a95 -0.009772081 0.4233682 -0.8965674 0.8991031
## 6033d90a5af2c702367b3a96 -0.183264035 0.5042589 -1.4222800 0.6993317
## 6034fc165af2c702367b3a98  0.472625690 0.5362046 -0.2480632 1.7230565
## 603500725af2c702367b3a99 -0.148338972 0.4889821 -1.2929017 0.7316093
## 603f84f15af2c702367b3a9b  0.006647246 0.4832838 -1.0250592 1.0672565
## 603f97625af2c702367b3a9d  0.492150415 0.5538176 -0.2471494 1.7797207
## 603fd5d95af2c702367b3a9e -0.032880360 0.4549867 -1.0192375 0.9214720
## 60409b7b5af2c702367b3a9f  0.247201765 0.4597769 -0.4899644 1.3432600
## 604b82b5a7718fbed181b336  0.406730552 0.5132121 -0.3002893 1.6122102
## 604f1239a7718fbed181b33f -0.087744781 0.5016136 -1.2894850 0.9229079
## 6050c1bf856f36729d2e5218 -0.160575634 0.4855568 -1.3590852 0.7268349
## 6050e1e7856f36729d2e5219  0.089035968 0.4328635 -0.7583526 1.0952367
## 6055fdc6856f36729d2e521b  0.143497995 0.4277343 -0.6298182 1.1584860
## 60579f2a856f36729d2e521e  0.014787560 0.5081688 -1.0773255 1.1335230
## 60589862856f36729d2e521f -0.306208419 0.5719094 -1.8334665 0.5513226
## 605a30a7856f36729d2e5221 -0.163348996 0.5434790 -1.5217667 0.7985603
## 605afa3a856f36729d2e5222 -0.307536062 0.5584935 -1.6880023 0.5397037
## 605c8bc6856f36729d2e5223 -0.007779202 0.4289889 -0.9133755 0.9047837
## 605f3f2d856f36729d2e5224  0.124312672 0.4261902 -0.6619304 1.1185785
## 605f46c3856f36729d2e5225 -0.294038602 0.5699904 -1.7610530 0.5595738
## 60605337856f36729d2e5226 -0.309975971 0.5806987 -1.7427612 0.5834909
## 60609ae6856f36729d2e5228 -0.084215278 0.4444429 -1.0653602 0.8066581
## 6061ce91856f36729d2e522e -0.154423119 0.4914238 -1.3714470 0.7165640
## 6061f106856f36729d2e5231 -0.295058208 0.5740861 -1.7538530 0.5843444
## 60672faa856f36729d2e523c  0.011448144 0.4805337 -1.0249230 1.0517728
## 6068ea9f856f36729d2e523e -0.006087948 0.4387570 -0.9178091 0.9529642
## 606db69d856f36729d2e5243 -0.147488289 0.5427777 -1.4462207 0.8502030
## 6075ab05856f36729d2e5247  0.060631922 0.4297414 -0.8000794 1.0382815

Sampling plots

plot(sonarqube_issues0.all, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues0.all, nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 17 rows containing missing values (geom_pointrange).

With experience predictor

As including all data points didn’t harm the model we will create this variant with all data points as well.

This variation includes work_experience_programming.s predictors as it can give further insight into how experience play a factor in the effect we try to measure. This is especially important as our sampling shewed towards containing less experienced developer than the population at large.

sonarqube_issues0.all.exp <- brm(
  "sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_experience_programming.s",
  prior = c(
    prior(normal(0, 1), class = "b"),
    prior(normal(1.5, 1), class = "Intercept"),
    prior(exponential(1), class = "sd"),
    prior(gamma(0.01, 0.01), class = "shape")
  ),
  family = negbinomial(),
  data = as.data.frame(d.completed),
  control = list(adapt_delta = 0.99),
  file = "fits/sonarqube_issues0.all.exp",
  file_refit = "on_change",
  seed = 20210421
)

Summary

summary(sonarqube_issues0.all.exp)

##  Family: negbinomial 
##   Links: mu = log; shape = identity 
## Formula: sonarqube_issues ~ 1 + high_debt_version + (1 | session) + work_experience_programming.s 
##    Data: as.data.frame(d.completed) (Number of observations: 51) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 29) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.49      0.33     0.02     1.20 1.01      822     1645
## 
## Population-Level Effects: 
##                               Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept                         0.73      0.30     0.13     1.33 1.00
## high_debt_versionfalse           -0.79      0.37    -1.53    -0.05 1.00
## work_experience_programming.s    -0.23      0.24    -0.69     0.24 1.00
##                               Bulk_ESS Tail_ESS
## Intercept                         2485     2801
## high_debt_versionfalse            5070     3178
## work_experience_programming.s     3670     2769
## 
## Family Specific Parameters: 
##       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## shape     1.42      3.54     0.39     5.06 1.00     1273     1349
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(sonarqube_issues0.all.exp)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033c6fc5af2c702367b3a93  0.147840184 0.4979331 -0.7641627 1.3383668
## 6033d69a5af2c702367b3a95 -0.028514199 0.4397984 -0.9508857 0.9083819
## 6033d90a5af2c702367b3a96 -0.222893202 0.5255988 -1.5317210 0.6663731
## 6034fc165af2c702367b3a98  0.446249367 0.5394912 -0.2934604 1.7279778
## 603500725af2c702367b3a99 -0.179509010 0.5118405 -1.4171940 0.7061523
## 603f84f15af2c702367b3a9b  0.003279425 0.5000172 -1.0484572 1.0788100
## 603f97625af2c702367b3a9d  0.462495121 0.5541805 -0.2828727 1.7689162
## 603fd5d95af2c702367b3a9e -0.056143645 0.4456638 -1.0427053 0.8794621
## 60409b7b5af2c702367b3a9f  0.209564983 0.4690933 -0.5812753 1.3288603
## 604b82b5a7718fbed181b336  0.394210612 0.5191392 -0.3369162 1.6383415
## 604f1239a7718fbed181b33f -0.095165696 0.5235878 -1.3408252 0.9029356
## 6050c1bf856f36729d2e5218 -0.133827049 0.4981417 -1.3173950 0.8170554
## 6050e1e7856f36729d2e5219  0.084352597 0.4274494 -0.7306554 1.0515933
## 6055fdc6856f36729d2e521b  0.113750087 0.4441904 -0.7066539 1.1653990
## 60579f2a856f36729d2e521e -0.002984136 0.5207445 -1.1323672 1.0848517
## 60589862856f36729d2e521f -0.248061220 0.5857061 -1.7050130 0.7029869
## 605a30a7856f36729d2e5221 -0.152530991 0.5446344 -1.4926320 0.8548997
## 605afa3a856f36729d2e5222 -0.267965857 0.5857895 -1.6980712 0.6801355
## 605c8bc6856f36729d2e5223  0.005183213 0.4525239 -0.9590902 0.9754727
## 605f3f2d856f36729d2e5224  0.344340924 0.6362826 -0.6192909 1.8391673
## 605f46c3856f36729d2e5225 -0.315157687 0.6149178 -1.8639652 0.5961203
## 60605337856f36729d2e5226 -0.329762122 0.6063929 -1.8746765 0.5832530
## 60609ae6856f36729d2e5228 -0.113227855 0.4685675 -1.1995080 0.7950688
## 6061ce91856f36729d2e522e -0.178054908 0.5170940 -1.4386847 0.7666872
## 6061f106856f36729d2e5231 -0.312922164 0.5765100 -1.7629610 0.5914138
## 60672faa856f36729d2e523c  0.004497664 0.4988850 -1.0396125 1.0910313
## 6068ea9f856f36729d2e523e -0.003998804 0.4298373 -0.9263463 0.9263328
## 606db69d856f36729d2e5243 -0.126652262 0.5471305 -1.4410468 0.9324305
## 6075ab05856f36729d2e5247  0.036304427 0.4425538 -0.8493844 1.0117328

Sampling plots

plot(sonarqube_issues0.all.exp, ask = FALSE)

Posterior predictive check

pp_check(sonarqube_issues0.all.exp, nsamples = 200, type = "bars") + xlim(-1, 15)

## Scale for 'x' is already present. Adding another scale for 'x', which will
## replace the existing scale.

## Warning: Removed 24 rows containing missing values (geom_pointrange).

Final model

• Fitting the model to all data point did not significantly damage the model and will be used as is a more fair representation of reality.
• Adding the experience predictors did not significantly damage the model and will be used as it provides useful insight.

This means that our final model, with all data points and experience predictors, is sonarqube_issues.all.exp

Interpreting the model

To begin interpreting the model we look at how it’s parameters were estimated. As our research is focused on how the outcome of the model is effected we will mainly analyze the \(\beta\) parameters.

\(\beta\) parameters

mcmc_areas(sonarqube_issues0.all.exp, pars = c("b_high_debt_versionfalse", "b_work_experience_programming.s"), prob = 0.95) + scale_y_discrete() +
  scale_y_discrete(labels=c("High debt version: false", "Professional programming experience")) +
  ggtitle("Beta parameters densities in time model", subtitle = "Shaded region marks 95% of the density. Line marks the median")

Effects sizes

We start by extracting posterior samples

scale_programming_experience <- function(x) {
  (x - mean(d.completed$work_experience_programming))/ sd(d.completed$work_experience_programming)
}
unscale_programming_experience <- function(x) {
  x * sd(d.completed$work_experience_programming) + mean(d.completed$work_experience_programming)
}

post_settings <- expand.grid(
  high_debt_version = c("false", "true"),
  session = NA,
  work_experience_programming.s = sapply(c(0, 3, 10, 25, 40), scale_programming_experience)
)

post <- posterior_predict(sonarqube_issues0.all.exp, newdata = post_settings) %>%
  melt(value.name = "estimate", varnames = c("sample_number", "settings_id")) %>%
  left_join(
    rowid_to_column(post_settings, var= "settings_id"),
    by = "settings_id"
  ) %>%
  mutate(work_experience_programming = unscale_programming_experience(work_experience_programming.s)) %>%
  select(
    estimate,
    high_debt_version,
    work_experience_programming
  )%>%
  mutate(estimate = estimate)

ggplot(post %>% filter(work_experience_programming == 10), aes(x=estimate, fill = high_debt_version)) +
  geom_bar(alpha = 0.5 , position = "dodge2") +
  scale_fill_manual(
    name = "Debt version",
    labels = c("Low debt", "High debt"),
    values = c("lightblue", "darkblue")
  ) +
  facet_grid(rows = vars(work_experience_programming)) +
  labs(
    title = "SonarQube issues generated",
    x = "Rate",
    y = "Incidence rate"
  ) + 
  xlim(-1, 10) + 
  scale_y_continuous(limits = NULL, breaks = c(500,1000,1500,2000,2500), labels = c("10%","20%","30%","40%","50%"))

## Warning: Removed 132 rows containing non-finite values (stat_count).

## Warning: Removed 1 rows containing missing values (geom_bar).

We can also plot the difference between good variable names for the high debt version and the low debt version.

ggplot(post, aes(x=estimate, fill = high_debt_version)) +
  geom_bar(alpha = 0.5 , position = "dodge2") +
  scale_fill_manual(
    name = "Debt version",
    labels = c("Low debt", "High debt"),
    values = c("lightblue", "darkblue")
  ) +
  facet_grid(rows = vars(work_experience_programming)) +
  labs(
    title = "SonarQube issues generated",
    x = "Rate",
    y = "Incidence rate"
  ) + 
  xlim(-1, 10) + 
  scale_y_continuous(limits = NULL, breaks = c(500,1000,1500,2000,2500), labels = c("10%","20%","30%","40%","50%"))

## Warning: Removed 772 rows containing non-finite values (stat_count).

## Warning: Removed 5 rows containing missing values (geom_bar).

post.diff <- post %>% filter(high_debt_version == "true")
post.diff$estimate = post.diff$estimate -  filter(post, high_debt_version == "false")$estimate

ggplot(post.diff, aes(x=estimate)) +
  geom_bar(quantile_lines = TRUE, quantile_fun = hdi, vline_linetype = 2) +
  xlim(-10, 10) +
  facet_grid(rows = vars(work_experience_programming)) +
  labs(
    title = "SonarQube issues generater / years of programming experience",
    subtitle = "Difference as: high debt issues - low debt issues",
    x = "Time (min)",
    y = "Density"
  )

mean(post.diff)

## Warning in mean.default(post.diff): argument is not numeric or logical:
## returning NA

## [1] NA